{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "settings_path = Path('../../settings/settings.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(str(settings_path), 'r') as file:\n",
    "    settings_dict = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'math_module': 'arithmetic__',\n",
       " 'train_level': '*',\n",
       " 'batch_size': 1024,\n",
       " 'thinking_steps': 0,\n",
       " 'epochs': 1,\n",
       " 'latent_dim': 256,\n",
       " 'embedding_dim': 256,\n",
       " 'save_path': '/artifacts/',\n",
       " 'data_path': '/storage/git/deep-math/data/'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "settings_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extrapolate  interpolate  train-easy  train-hard  train-medium\r\n"
     ]
    }
   ],
   "source": [
    "raw_path = Path('/storage/git/deep-math/data/raw/v1.0/')\n",
    "!ls {raw_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "fastai_data_folder = Path('/storage/git/deep-math/data/interim/fastai/')\n",
    "fastai_data_folder.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extrapolate algebra__polynomial_roots_big.txt\n",
      "extrapolate arithmetic__add_or_sub_big.txt\n",
      "extrapolate arithmetic__add_sub_multiple_longer.txt\n",
      "extrapolate arithmetic__div_big.txt\n",
      "extrapolate arithmetic__mixed_longer.txt\n",
      "extrapolate arithmetic__mul_big.txt\n",
      "extrapolate arithmetic__mul_div_multiple_longer.txt\n",
      "extrapolate comparison__closest_more.txt\n",
      "extrapolate comparison__kth_biggest_more.txt\n",
      "extrapolate comparison__sort_more.txt\n",
      "extrapolate measurement__conversion.txt\n",
      "extrapolate numbers__place_value_big.txt\n",
      "extrapolate numbers__round_number_big.txt\n",
      "extrapolate probability__swr_p_level_set_more_samples.txt\n",
      "extrapolate probability__swr_p_sequence_more_samples.txt\n",
      "interpolate algebra__linear_1d.txt\n",
      "interpolate algebra__linear_1d_composed.txt\n",
      "interpolate algebra__linear_2d.txt\n",
      "interpolate algebra__linear_2d_composed.txt\n",
      "interpolate algebra__polynomial_roots.txt\n",
      "interpolate algebra__polynomial_roots_composed.txt\n",
      "interpolate algebra__sequence_next_term.txt\n",
      "interpolate algebra__sequence_nth_term.txt\n",
      "interpolate arithmetic__add_or_sub.txt\n",
      "interpolate arithmetic__add_or_sub_in_base.txt\n",
      "interpolate arithmetic__add_sub_multiple.txt\n",
      "interpolate arithmetic__div.txt\n",
      "interpolate arithmetic__mixed.txt\n",
      "interpolate arithmetic__mul.txt\n",
      "interpolate arithmetic__mul_div_multiple.txt\n",
      "interpolate arithmetic__nearest_integer_root.txt\n",
      "interpolate arithmetic__simplify_surd.txt\n",
      "interpolate calculus__differentiate.txt\n",
      "interpolate calculus__differentiate_composed.txt\n",
      "interpolate comparison__closest.txt\n",
      "interpolate comparison__closest_composed.txt\n",
      "interpolate comparison__kth_biggest.txt\n",
      "interpolate comparison__kth_biggest_composed.txt\n",
      "interpolate comparison__pair.txt\n",
      "interpolate comparison__pair_composed.txt\n",
      "interpolate comparison__sort.txt\n",
      "interpolate comparison__sort_composed.txt\n",
      "interpolate measurement__conversion.txt\n",
      "interpolate measurement__time.txt\n",
      "interpolate numbers__base_conversion.txt\n",
      "interpolate numbers__div_remainder.txt\n",
      "interpolate numbers__div_remainder_composed.txt\n",
      "interpolate numbers__gcd.txt\n",
      "interpolate numbers__gcd_composed.txt\n",
      "interpolate numbers__is_factor.txt\n",
      "interpolate numbers__is_factor_composed.txt\n",
      "interpolate numbers__is_prime.txt\n",
      "interpolate numbers__is_prime_composed.txt\n",
      "interpolate numbers__lcm.txt\n",
      "interpolate numbers__lcm_composed.txt\n",
      "interpolate numbers__list_prime_factors.txt\n",
      "interpolate numbers__list_prime_factors_composed.txt\n",
      "interpolate numbers__place_value.txt\n",
      "interpolate numbers__place_value_composed.txt\n",
      "interpolate numbers__round_number.txt\n",
      "interpolate numbers__round_number_composed.txt\n",
      "interpolate polynomials__add.txt\n",
      "interpolate polynomials__coefficient_named.txt\n",
      "interpolate polynomials__collect.txt\n",
      "interpolate polynomials__compose.txt\n",
      "interpolate polynomials__evaluate.txt\n",
      "interpolate polynomials__evaluate_composed.txt\n",
      "interpolate polynomials__expand.txt\n",
      "interpolate polynomials__simplify_power.txt\n",
      "interpolate probability__swr_p_level_set.txt\n",
      "interpolate probability__swr_p_sequence.txt\n",
      "train-easy algebra__linear_1d.txt\n",
      "train-easy algebra__linear_1d_composed.txt\n",
      "train-easy algebra__linear_2d.txt\n",
      "train-easy algebra__linear_2d_composed.txt\n",
      "train-easy algebra__polynomial_roots.txt\n",
      "train-easy algebra__polynomial_roots_composed.txt\n",
      "train-easy algebra__sequence_next_term.txt\n",
      "train-easy algebra__sequence_nth_term.txt\n",
      "train-easy arithmetic__add_or_sub.txt\n",
      "train-easy arithmetic__add_or_sub_in_base.txt\n",
      "train-easy arithmetic__add_sub_multiple.txt\n",
      "train-easy arithmetic__div.txt\n",
      "train-easy arithmetic__mixed.txt\n",
      "train-easy arithmetic__mul.txt\n",
      "train-easy arithmetic__mul_div_multiple.txt\n",
      "train-easy arithmetic__nearest_integer_root.txt\n",
      "train-easy arithmetic__simplify_surd.txt\n",
      "train-easy calculus__differentiate.txt\n",
      "train-easy calculus__differentiate_composed.txt\n",
      "train-easy comparison__closest.txt\n",
      "train-easy comparison__closest_composed.txt\n",
      "train-easy comparison__kth_biggest.txt\n",
      "train-easy comparison__kth_biggest_composed.txt\n",
      "train-easy comparison__pair.txt\n",
      "train-easy comparison__pair_composed.txt\n",
      "train-easy comparison__sort.txt\n",
      "train-easy comparison__sort_composed.txt\n",
      "train-easy measurement__conversion.txt\n",
      "train-easy measurement__time.txt\n",
      "train-easy numbers__base_conversion.txt\n",
      "train-easy numbers__div_remainder.txt\n",
      "train-easy numbers__div_remainder_composed.txt\n",
      "train-easy numbers__gcd.txt\n",
      "train-easy numbers__gcd_composed.txt\n",
      "train-easy numbers__is_factor.txt\n",
      "train-easy numbers__is_factor_composed.txt\n",
      "train-easy numbers__is_prime.txt\n",
      "train-easy numbers__is_prime_composed.txt\n",
      "train-easy numbers__lcm.txt\n",
      "train-easy numbers__lcm_composed.txt\n",
      "train-easy numbers__list_prime_factors.txt\n",
      "train-easy numbers__list_prime_factors_composed.txt\n",
      "train-easy numbers__place_value.txt\n",
      "train-easy numbers__place_value_composed.txt\n",
      "train-easy numbers__round_number.txt\n",
      "train-easy numbers__round_number_composed.txt\n",
      "train-easy polynomials__add.txt\n",
      "train-easy polynomials__coefficient_named.txt\n",
      "train-easy polynomials__collect.txt\n",
      "train-easy polynomials__compose.txt\n",
      "train-easy polynomials__evaluate.txt\n",
      "train-easy polynomials__evaluate_composed.txt\n",
      "train-easy polynomials__expand.txt\n",
      "train-easy polynomials__simplify_power.txt\n",
      "train-easy probability__swr_p_level_set.txt\n",
      "train-easy probability__swr_p_sequence.txt\n",
      "train-hard algebra__linear_1d.txt\n",
      "train-hard algebra__linear_1d_composed.txt\n",
      "train-hard algebra__linear_2d.txt\n",
      "train-hard algebra__linear_2d_composed.txt\n",
      "train-hard algebra__polynomial_roots.txt\n",
      "train-hard algebra__polynomial_roots_composed.txt\n",
      "train-hard algebra__sequence_next_term.txt\n",
      "train-hard algebra__sequence_nth_term.txt\n",
      "train-hard arithmetic__add_or_sub.txt\n",
      "train-hard arithmetic__add_or_sub_in_base.txt\n",
      "train-hard arithmetic__add_sub_multiple.txt\n",
      "train-hard arithmetic__div.txt\n",
      "train-hard arithmetic__mixed.txt\n",
      "train-hard arithmetic__mul.txt\n",
      "train-hard arithmetic__mul_div_multiple.txt\n",
      "train-hard arithmetic__nearest_integer_root.txt\n",
      "train-hard arithmetic__simplify_surd.txt\n",
      "train-hard calculus__differentiate.txt\n",
      "train-hard calculus__differentiate_composed.txt\n",
      "train-hard comparison__closest.txt\n",
      "train-hard comparison__closest_composed.txt\n",
      "train-hard comparison__kth_biggest.txt\n",
      "train-hard comparison__kth_biggest_composed.txt\n",
      "train-hard comparison__pair.txt\n",
      "train-hard comparison__pair_composed.txt\n",
      "train-hard comparison__sort.txt\n",
      "train-hard comparison__sort_composed.txt\n",
      "train-hard measurement__conversion.txt\n",
      "train-hard measurement__time.txt\n",
      "train-hard numbers__base_conversion.txt\n",
      "train-hard numbers__div_remainder.txt\n",
      "train-hard numbers__div_remainder_composed.txt\n",
      "train-hard numbers__gcd.txt\n",
      "train-hard numbers__gcd_composed.txt\n",
      "train-hard numbers__is_factor.txt\n",
      "train-hard numbers__is_factor_composed.txt\n",
      "train-hard numbers__is_prime.txt\n",
      "train-hard numbers__is_prime_composed.txt\n",
      "train-hard numbers__lcm.txt\n",
      "train-hard numbers__lcm_composed.txt\n",
      "train-hard numbers__list_prime_factors.txt\n",
      "train-hard numbers__list_prime_factors_composed.txt\n",
      "train-hard numbers__place_value.txt\n",
      "train-hard numbers__place_value_composed.txt\n",
      "train-hard numbers__round_number.txt\n",
      "train-hard numbers__round_number_composed.txt\n",
      "train-hard polynomials__add.txt\n",
      "train-hard polynomials__coefficient_named.txt\n",
      "train-hard polynomials__collect.txt\n",
      "train-hard polynomials__compose.txt\n",
      "train-hard polynomials__evaluate.txt\n",
      "train-hard polynomials__evaluate_composed.txt\n",
      "train-hard polynomials__expand.txt\n",
      "train-hard polynomials__simplify_power.txt\n",
      "train-hard probability__swr_p_level_set.txt\n",
      "train-hard probability__swr_p_sequence.txt\n",
      "train-medium algebra__linear_1d.txt\n",
      "train-medium algebra__linear_1d_composed.txt\n",
      "train-medium algebra__linear_2d.txt\n",
      "train-medium algebra__linear_2d_composed.txt\n",
      "train-medium algebra__polynomial_roots.txt\n",
      "train-medium algebra__polynomial_roots_composed.txt\n",
      "train-medium algebra__sequence_next_term.txt\n",
      "train-medium algebra__sequence_nth_term.txt\n",
      "train-medium arithmetic__add_or_sub.txt\n",
      "train-medium arithmetic__add_or_sub_in_base.txt\n",
      "train-medium arithmetic__add_sub_multiple.txt\n",
      "train-medium arithmetic__div.txt\n",
      "train-medium arithmetic__mixed.txt\n",
      "train-medium arithmetic__mul.txt\n",
      "train-medium arithmetic__mul_div_multiple.txt\n",
      "train-medium arithmetic__nearest_integer_root.txt\n",
      "train-medium arithmetic__simplify_surd.txt\n",
      "train-medium calculus__differentiate.txt\n",
      "train-medium calculus__differentiate_composed.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train-medium comparison__closest.txt\n",
      "train-medium comparison__closest_composed.txt\n",
      "train-medium comparison__kth_biggest.txt\n",
      "train-medium comparison__kth_biggest_composed.txt\n",
      "train-medium comparison__pair.txt\n",
      "train-medium comparison__pair_composed.txt\n",
      "train-medium comparison__sort.txt\n",
      "train-medium comparison__sort_composed.txt\n",
      "train-medium measurement__conversion.txt\n",
      "train-medium measurement__time.txt\n",
      "train-medium numbers__base_conversion.txt\n",
      "train-medium numbers__div_remainder.txt\n",
      "train-medium numbers__div_remainder_composed.txt\n",
      "train-medium numbers__gcd.txt\n",
      "train-medium numbers__gcd_composed.txt\n",
      "train-medium numbers__is_factor.txt\n",
      "train-medium numbers__is_factor_composed.txt\n",
      "train-medium numbers__is_prime.txt\n",
      "train-medium numbers__is_prime_composed.txt\n",
      "train-medium numbers__lcm.txt\n",
      "train-medium numbers__lcm_composed.txt\n",
      "train-medium numbers__list_prime_factors.txt\n",
      "train-medium numbers__list_prime_factors_composed.txt\n",
      "train-medium numbers__place_value.txt\n",
      "train-medium numbers__place_value_composed.txt\n",
      "train-medium numbers__round_number.txt\n",
      "train-medium numbers__round_number_composed.txt\n",
      "train-medium polynomials__add.txt\n",
      "train-medium polynomials__coefficient_named.txt\n",
      "train-medium polynomials__collect.txt\n",
      "train-medium polynomials__compose.txt\n",
      "train-medium polynomials__evaluate.txt\n",
      "train-medium polynomials__evaluate_composed.txt\n",
      "train-medium polynomials__expand.txt\n",
      "train-medium polynomials__simplify_power.txt\n",
      "train-medium probability__swr_p_level_set.txt\n",
      "train-medium probability__swr_p_sequence.txt\n"
     ]
    }
   ],
   "source": [
    "thinking_steps = 16\n",
    "\n",
    "folders = !ls {raw_path}\n",
    "for folder in folders:\n",
    "    \n",
    "    save_path = fastai_data_folder/folder\n",
    "    save_path.mkdir(exist_ok=True)\n",
    "    \n",
    "    files = !ls {raw_path/folder}\n",
    "    for file in files:\n",
    "        print(folder, file)\n",
    "        file_path = raw_path/(folder + '/' + file)\n",
    "        \n",
    "        with open(str(file_path), 'r', encoding='utf-8') as f:\n",
    "            lines = f.read().split('\\n')[:-1]\n",
    "\n",
    "        questions = lines[0::2]\n",
    "        answers = lines[1::2]\n",
    "        \n",
    "        train_text = [question + ' '*thinking_steps + answer + '\\n' for question, answer in zip(questions, answers)]\n",
    "        output_txt = ''.join(train_text)\n",
    "        \n",
    "        with open(str(save_path/file),'w', encoding='utf-8') as f:\n",
    "            f.write(output_txt)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
